{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ignored-upper",
   "metadata": {},
   "source": [
    "## Istio Notebook Examples\n",
    "\n",
    "Assumes\n",
    "\n",
    " * You have installed istio as per their docs\n",
    " * You have exposed the ingressgateway as an external loadbalancer\n",
    " \n",
    " \n",
    "tested with:\n",
    "\n",
    "```\n",
    "istioctl version\n",
    "1.13.2\n",
    "\n",
    "istioctl install --set profile=demo -y\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "arabic-april",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'172.21.255.1'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "INGRESS_IP=!kubectl get svc istio-ingressgateway -n istio-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}'\n",
    "INGRESS_IP=INGRESS_IP[0]\n",
    "import os\n",
    "os.environ['INGRESS_IP'] = INGRESS_IP\n",
    "INGRESS_IP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "excessive-garlic",
   "metadata": {},
   "source": [
    "### Istio Single Model Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "present-lightning",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n",
      "  storageUri: gs://seldon-models/mlserver/iris\r\n",
      "---\r\n",
      "apiVersion: networking.istio.io/v1beta1\r\n",
      "kind: Gateway\r\n",
      "metadata:\r\n",
      "  name: seldon-gateway\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  selector:\r\n",
      "    app: istio-ingressgateway\r\n",
      "    istio: ingressgateway\r\n",
      "  servers:\r\n",
      "  - hosts:\r\n",
      "    - '*'\r\n",
      "    port:\r\n",
      "      name: http\r\n",
      "      number: 80\r\n",
      "      protocol: HTTP\r\n",
      "  - hosts:\r\n",
      "    - '*'\r\n",
      "    port:\r\n",
      "      name: https\r\n",
      "      number: 443\r\n",
      "      protocol: HTTPS\r\n",
      "    tls:\r\n",
      "      mode: SIMPLE\r\n",
      "      privateKey: /etc/istio/ingressgateway-certs/tls.key\r\n",
      "      serverCertificate: /etc/istio/ingressgateway-certs/tls.crt\r\n",
      "---\r\n",
      "apiVersion: networking.istio.io/v1beta1\r\n",
      "kind: VirtualService\r\n",
      "metadata:\r\n",
      "  name: iris-route\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  gateways:\r\n",
      "  - istio-system/seldon-gateway\r\n",
      "  hosts:\r\n",
      "  - '*'\r\n",
      "  http:\r\n",
      "  - match:\r\n",
      "    - uri:\r\n",
      "        prefix: /v2\r\n",
      "    name: iris-http\r\n",
      "    route:\r\n",
      "    - destination:\r\n",
      "        host: seldon-mesh.seldon-mesh.svc.cluster.local\r\n",
      "      headers:\r\n",
      "        request:\r\n",
      "          set:\r\n",
      "            seldon-model: iris\r\n",
      "  - match:\r\n",
      "    - uri:\r\n",
      "        prefix: /inference.GRPCInferenceService\r\n",
      "    name: iris-grpc\r\n",
      "    route:\r\n",
      "    - destination:\r\n",
      "        host: seldon-mesh.seldon-mesh.svc.cluster.local\r\n",
      "      headers:\r\n",
      "        request:\r\n",
      "          set:\r\n",
      "            seldon-model: iris\r\n"
     ]
    }
   ],
   "source": [
    "!kustomize build config/single-model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "satellite-canadian",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/iris unchanged\r\n",
      "gateway.networking.istio.io/seldon-gateway unchanged\r\n",
      "virtualservice.networking.istio.io/iris-route configured\r\n"
     ]
    }
   ],
   "source": [
    "!kustomize build config/single-model | kubectl apply -f -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "silent-objective",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/iris condition met\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n seldon-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "clean-primary",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*   Trying 172.21.255.1...\n",
      "* Connected to 172.21.255.1 (172.21.255.1) port 80 (#0)\n",
      "> POST /v2/models/iris/infer HTTP/1.1\n",
      "> Host: 172.21.255.1\n",
      "> User-Agent: curl/7.47.0\n",
      "> Accept: */*\n",
      "> Content-Type: application/json\n",
      "> Content-Length: 94\n",
      "> \n",
      "* upload completely sent off: 94 out of 94 bytes\n",
      "< HTTP/1.1 200 OK\n",
      "< content-length: 196\n",
      "< content-type: application/json\n",
      "< date: Sat, 16 Apr 2022 15:34:11 GMT\n",
      "< server: istio-envoy\n",
      "< x-envoy-upstream-service-time: 802\n",
      "< seldon-route: iris_1\n",
      "< \n",
      "* Connection #0 to host 172.21.255.1 left intact\n",
      "{\"model_name\":\"iris_1\",\"model_version\":\"1\",\"id\":\"83520c4a-c7f1-4363-9bfd-60c5d8ee2dc5\",\"parameters\":null,\"outputs\":[{\"name\":\"predict\",\"shape\":[1],\"datatype\":\"INT64\",\"parameters\":null,\"data\":[2]}]}"
     ]
    }
   ],
   "source": [
    "!curl -v http://${INGRESS_IP}/v2/models/iris/infer -H \"Content-Type: application/json\" \\\n",
    "        -d '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "greek-penny",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"modelName\": \"iris_1\",\r\n",
      "  \"modelVersion\": \"1\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"datatype\": \"INT64\",\r\n",
      "      \"shape\": [\r\n",
      "        \"1\"\r\n",
      "      ],\r\n",
      "      \"contents\": {\r\n",
      "        \"int64Contents\": [\r\n",
      "          \"2\"\r\n",
      "        ]\r\n",
      "      }\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!grpcurl -d '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' \\\n",
    "    -plaintext \\\n",
    "    -import-path ../../apis \\\n",
    "    -proto ../../apis/mlops/v2_dataplane/v2_dataplane.proto \\\n",
    "    ${INGRESS_IP}:80 inference.GRPCInferenceService/ModelInfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "durable-profession",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io \"iris\" deleted\n",
      "gateway.networking.istio.io \"seldon-gateway\" deleted\n",
      "virtualservice.networking.istio.io \"iris-route\" deleted\n"
     ]
    }
   ],
   "source": [
    "!kustomize build config/single-model | kubectl delete -f -"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vulnerable-enterprise",
   "metadata": {},
   "source": [
    "### Traffic Split Two Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "practical-russia",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris1\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n",
      "  storageUri: gs://seldon-models/mlserver/iris\r\n",
      "---\r\n",
      "apiVersion: mlops.seldon.io/v1alpha1\r\n",
      "kind: Model\r\n",
      "metadata:\r\n",
      "  name: iris2\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  requirements:\r\n",
      "  - sklearn\r\n",
      "  storageUri: gs://seldon-models/mlserver/iris\r\n",
      "---\r\n",
      "apiVersion: networking.istio.io/v1beta1\r\n",
      "kind: Gateway\r\n",
      "metadata:\r\n",
      "  name: seldon-gateway\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  selector:\r\n",
      "    app: istio-ingressgateway\r\n",
      "    istio: ingressgateway\r\n",
      "  servers:\r\n",
      "  - hosts:\r\n",
      "    - '*'\r\n",
      "    port:\r\n",
      "      name: http\r\n",
      "      number: 80\r\n",
      "      protocol: HTTP\r\n",
      "---\r\n",
      "apiVersion: networking.istio.io/v1beta1\r\n",
      "kind: VirtualService\r\n",
      "metadata:\r\n",
      "  name: iris-route\r\n",
      "  namespace: seldon-mesh\r\n",
      "spec:\r\n",
      "  gateways:\r\n",
      "  - seldon-gateway\r\n",
      "  hosts:\r\n",
      "  - '*'\r\n",
      "  http:\r\n",
      "  - match:\r\n",
      "    - uri:\r\n",
      "        prefix: /v2/models/iris\r\n",
      "    name: iris-http\r\n",
      "    route:\r\n",
      "    - destination:\r\n",
      "        host: seldon-mesh.seldon-mesh.svc.cluster.local\r\n",
      "      headers:\r\n",
      "        request:\r\n",
      "          set:\r\n",
      "            seldon-model: iris1\r\n",
      "      weight: 50\r\n",
      "    - destination:\r\n",
      "        host: seldon-mesh.seldon-mesh.svc.cluster.local\r\n",
      "      headers:\r\n",
      "        request:\r\n",
      "          set:\r\n",
      "            seldon-model: iris2\r\n",
      "      weight: 50\r\n",
      "  - match:\r\n",
      "    - uri:\r\n",
      "        prefix: /inference.GRPCInferenceService\r\n",
      "    name: iris-grpc\r\n",
      "    route:\r\n",
      "    - destination:\r\n",
      "        host: seldon-mesh.seldon-mesh.svc.cluster.local\r\n",
      "      headers:\r\n",
      "        request:\r\n",
      "          set:\r\n",
      "            seldon-model: iris1\r\n",
      "      weight: 50\r\n",
      "    - destination:\r\n",
      "        host: seldon-mesh.seldon-mesh.svc.cluster.local\r\n",
      "      headers:\r\n",
      "        request:\r\n",
      "          set:\r\n",
      "            seldon-model: iris2\r\n",
      "      weight: 50\r\n"
     ]
    }
   ],
   "source": [
    "!kustomize build config/traffic-split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "meaningful-toilet",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/iris1 created\n",
      "model.mlops.seldon.io/iris2 created\n",
      "gateway.networking.istio.io/seldon-gateway created\n",
      "virtualservice.networking.istio.io/iris-route created\n"
     ]
    }
   ],
   "source": [
    "!kustomize build config/traffic-split | kubectl apply -f -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "sophisticated-junior",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io/iris1 condition met\r\n",
      "model.mlops.seldon.io/iris2 condition met\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl wait --for condition=ready --timeout=300s model --all -n seldon-mesh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "worthy-donna",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*   Trying 172.21.255.1...\n",
      "* Connected to 172.21.255.1 (172.21.255.1) port 80 (#0)\n",
      "> POST /v2/models/iris/infer HTTP/1.1\n",
      "> Host: 172.21.255.1\n",
      "> User-Agent: curl/7.47.0\n",
      "> Accept: */*\n",
      "> Content-Type: application/json\n",
      "> Content-Length: 94\n",
      "> \n",
      "* upload completely sent off: 94 out of 94 bytes\n",
      "< HTTP/1.1 200 OK\n",
      "< content-length: 197\n",
      "< content-type: application/json\n",
      "< date: Sat, 16 Apr 2022 15:35:01 GMT\n",
      "< server: istio-envoy\n",
      "< x-envoy-upstream-service-time: 801\n",
      "< seldon-route: iris1_1\n",
      "< \n",
      "* Connection #0 to host 172.21.255.1 left intact\n",
      "{\"model_name\":\"iris1_1\",\"model_version\":\"1\",\"id\":\"b54e6d8c-d253-4bb9-bb64-02c2ee49e89f\",\"parameters\":null,\"outputs\":[{\"name\":\"predict\",\"shape\":[1],\"datatype\":\"INT64\",\"parameters\":null,\"data\":[2]}]}"
     ]
    }
   ],
   "source": [
    "!curl -v http://${INGRESS_IP}/v2/models/iris/infer -H \"Content-Type: application/json\" \\\n",
    "        -d '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "extensive-importance",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "  \"modelName\": \"iris1_1\",\r\n",
      "  \"modelVersion\": \"1\",\r\n",
      "  \"outputs\": [\r\n",
      "    {\r\n",
      "      \"name\": \"predict\",\r\n",
      "      \"datatype\": \"INT64\",\r\n",
      "      \"shape\": [\r\n",
      "        \"1\"\r\n",
      "      ],\r\n",
      "      \"contents\": {\r\n",
      "        \"int64Contents\": [\r\n",
      "          \"2\"\r\n",
      "        ]\r\n",
      "      }\r\n",
      "    }\r\n",
      "  ]\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!grpcurl -d '{\"model_name\":\"iris1\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' \\\n",
    "    -plaintext \\\n",
    "    -import-path ../../apis \\\n",
    "    -proto ../../apis/mlops/v2_dataplane/v2_dataplane.proto \\\n",
    "    ${INGRESS_IP}:80 inference.GRPCInferenceService/ModelInfer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "specific-american",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.mlops.seldon.io \"iris1\" deleted\r\n",
      "model.mlops.seldon.io \"iris2\" deleted\r\n",
      "gateway.networking.istio.io \"seldon-gateway\" deleted\r\n",
      "virtualservice.networking.istio.io \"iris-route\" deleted\r\n"
     ]
    }
   ],
   "source": [
    "!kustomize build config/traffic-split | kubectl delete -f -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de7c89f6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
